# coding=utf-8
from gensim import corpora, models, similarities

documents = ["Human machine interface for lab abc computer applications",
              "A survey of user opinion of computer system response time",
              "The EPS user interface management system",
              "System and human system engineering testing of EPS",
              "Relation of user perceived response time to error measurement",
              "The generation of random binary unordered trees",
              "The intersection graph of paths in trees",
              "Graph minors IV Widths of trees and well quasi ordering",
              "Graph minors A survey"]

# 去除停用词并分词
# 译者注：这里只是例子，实际上还有其他停用词
#         处理中文时，请借助 Py结巴分词 https://github.com/fxsjy/jieba
stoplist = set('for a of the and to in'.split())
texts = [[word for word in document.lower().split() if word not in stoplist]
          for document in documents]
#print texts

# 去除仅出现一次的单词
from collections import defaultdict
frequency = defaultdict(int)
for text in texts:
    for token in text:
        frequency[token] += 1

texts = [[token for token in text if frequency[token] > 1]
        for text in texts]
# 打印最后的这个特征库
from pprint import pprint   # pretty-printer
print 'texts'
pprint(texts)

# 保存字典
dictionary = corpora.Dictionary(texts)
dictionary.save('deerwester.dict') # 把字典保存起来，方便以后使用
print "dictionary"
print(dictionary)
print(dictionary.token2id)

# 产生稀疏文档向量
# 稀疏向量[(0, 1), (1, 1)]表示：在“Human computer interaction”中“computer”(id 0) 和“human”(id 1)各出现一次，其他10个dictionary中的单词没有出现过（隐含的）
new_doc = "Human computer interaction"
new_vec = dictionary.doc2bow(new_doc.lower().split())   # 对每个不同单词的出现次数进行了计数，并将单词转换为其编号，然后以稀疏向量的形式返回结果
print(new_vec) # "interaction"没有在dictionary中出现，因此忽略
corpus = [dictionary.doc2bow(text) for text in texts]
corpora.MmCorpus.serialize('deerwester.mm', corpus) # 存入硬盘，以备后需
print "corpus"
print(corpus)

class MyCorpus(object):
    def __iter__(self):
        for line in open('mycorpus.txt'):
            # assume there's one document per line, tokens separated by whitespace
            yield dictionary.doc2bow(line.lower().split())

corpus_memory_friendly = MyCorpus() # 没有将整个语料库载入内存
print(corpus_memory_friendly)
for vector in corpus_memory_friendly: # 一次读入内存一个向量
    print(vector)


# 收集所有符号的统计信息
dictionary = corpora.Dictionary(line.lower().split() for line in open('mycorpus.txt'))
# 收集停用词和仅出现一次的词的id
stop_ids = [dictionary.token2id[stopword] for stopword in stoplist
             if stopword in dictionary.token2id]
once_ids = [tokenid for tokenid, docfreq in dictionary.dfs.iteritems() if docfreq == 1]
dictionary.filter_tokens(stop_ids + once_ids) # 删除停用词和仅出现一次的词
dictionary.compactify() # 消除id序列在删除词后产生的不连续的缺口
print "read from text dictionary"
print(dictionary)


# 将语料库全部导入内存的方法
# print(list(corpus)) # 调用list()将会把所有的序列转换为普通Python List





